import requests
from bs4 import BeautifulSoup
import mysql.connector

# 创建MySQL数据库连接
conn = mysql.connector.connect(
    host="localhost",
    user="yourusername",
    password="yourpassword",
    database="yourdatabase"
)

# 创建游标对象
cursor = conn.cursor()

# 创建表
cursor.execute('''CREATE TABLE IF NOT EXISTS projects
                  (name VARCHAR(255), description TEXT, url VARCHAR(255))''')

# 循环爬取多个页面
for page in range(1, 1294):
    # 发起GET请求
    url = f'https://gitee.com/organizations/mirrors/projects?page={page}'
    response = requests.get(url)

    # 解析HTML
    soup = BeautifulSoup(response.text, 'html.parser')

    # 找到所有项目元素
    projects = soup.find_all('div', class_='item project-item')

    # 遍历项目列表
    for project in projects:
        # 获取项目名、简介和项目地址
        name = project.find('a', class_='project-name').text.strip()
        description = project.find('p', class_='description').text.strip()
        url = project.find('a', class_='project-name')['href']

        # 插入数据到数据库
        cursor.execute("INSERT INTO projects (name, description, url) VALUES (%s, %s, %s)",
                       (name, description, url))

    # 提交更改
    conn.commit()

# 关闭连接
conn.close()
